{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0894a735",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig\n",
    "import torch\n",
    "\n",
    "TORCH_DTYPE = 'bfloat16'\n",
    "nf4_config = BitsAndBytesConfig(\n",
    "    load_in_4bit=True,\n",
    "    bnb_4bit_quant_type='nf4',\n",
    "    bnb_4bit_use_double_quant=True,\n",
    "    bnb_4bit_compute_dtype=getattr(torch, TORCH_DTYPE)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6daae66f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bc09a61b9a5441f88474ea7f06399839",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)okenizer_config.json:   0%|          | 0.00/953 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2b1390cbf2c1421d960dfa6b667627fe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading tokenizer.model:   0%|          | 0.00/493k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3ff6786eccb54afe86ae8d11440b8c59",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)/main/tokenizer.json:   0%|          | 0.00/1.80M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b973fabaddc64915987f0646576178b5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)in/added_tokens.json:   0%|          | 0.00/42.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d32d7da0c38b41f2b7ffa9a480cb9b47",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)cial_tokens_map.json:   0%|          | 0.00/145 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained('mesolitica/mistral-7b-32768-fpf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0da5df04",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1985b8a34f7c4c1e805ddc5603dae3df",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)lve/main/config.json:   0%|          | 0.00/615 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "91955a557e75472caaf224c8d8c9be0b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)fetensors.index.json:   0%|          | 0.00/23.9k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5d6f3da545a445ce9a05706a3ccf6d90",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a10f46fb4fbb4e898a9276a5430c1438",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)of-00002.safetensors:   0%|          | 0.00/9.94G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ecf549c9e7bf43cc8e94bef4a598b23d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)of-00002.safetensors:   0%|          | 0.00/4.54G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "34d08096b1974cc18b463509a30c1db1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3a76ec32cb6146efb82fff653ee44c00",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)neration_config.json:   0%|          | 0.00/111 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model = AutoModelForCausalLM.from_pretrained(\n",
    "    'mesolitica/mistral-7b-32768-fpf',\n",
    "    use_flash_attention_2 = True,\n",
    "    quantization_config = nf4_config\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "07881cfc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': tensor([[    1,  1537,   282,   276, 28747,   524, 28780,  3701,  8582,   979,\n",
       "         28708, 28804,    13, 28798,  1067,  4209, 28747]], device='cuda:0'), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], device='cuda:0')}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prompt = '<s>Soalan: KWSP tu apa?\\nJawapan:'\n",
    "inputs = tokenizer([prompt], return_tensors='pt', add_special_tokens=False).to('cuda')\n",
    "inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "67d3c231",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<s> Soalan: KWSP tu apa?\n",
      "Jawapan: kwsp ialah kwsp iaitu kumpulan wang simpanan pekerja</s>\n"
     ]
    }
   ],
   "source": [
    "generate_kwargs = dict(\n",
    "    inputs,\n",
    "    max_new_tokens=512,\n",
    "    top_p=0.95,\n",
    "    top_k=50,\n",
    "    temperature=0.9,\n",
    "    do_sample=True,\n",
    "    num_beams=1,\n",
    "    repetition_penalty=1.05,\n",
    ")\n",
    "r = model.generate(**generate_kwargs)\n",
    "print(tokenizer.decode(r[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4efaf03f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': tensor([[    1,  1537,   282,   276, 28747,   446,   269, 10405,  6125,   748,\n",
       "           515,   519,  2117,   290,   491,   276,   307,  8608,   462, 28719,\n",
       "           491, 28804,    13, 28798,  1067,  4209, 28747]], device='cuda:0'), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "         1, 1, 1]], device='cuda:0')}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prompt = '<s>Soalan: kenapa malaysia suka makan nasi lemak?\\nJawapan:'\n",
    "inputs = tokenizer([prompt], return_tensors='pt', add_special_tokens=False).to('cuda')\n",
    "inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bdc41dda",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<s> Soalan: kenapa malaysia suka makan nasi lemak?\n",
      "Jawapan:Nasi Lemak merupakan antara sarapan pagi yang paling popular dalam kalangan rakyat Malaysia terutamanya orang Melayu. Ia mudah didapati di mana-mana sahaja seperti restoran mamak, gerai tepi jalan, gerai di pasar pagi, di stesen minyak, pasar malam, dan sebagainya. Sajian ini juga dikenali sebagai Nasi lemak antarabangsa kerana mendapat sambutan hangat daripada seluruh dunia. Selain itu, nasi lemak mempunyai bau yang sangat menyelerakan dan rasanya juga sangat unik. Tidak hairanlah mengapa ia digemari oleh ramai orang.Nasi lemak mempunyai santan, kerisik, dan serbuk kunyit yang ditumis dalam lemak kelapa dan dibungkus dalam daun pisang atau mangkuk plastik. Nasi lemak biasanya disajikan bersama ayam goreng, daging rendang, telur rebus, ikan bilis, timun, dan sambal yang dibuat daripada kacang tanah, udang atau petai. Variasi lain termasuk parutan kelapa muda, bilis goreng, udang goreng, dan sotong goreng. Di Malaysia, nasi lemak boleh didapati di gerai di jalanan, restoran tradisional, gerai di pasar pagi, dan di gerai tepi jalan di seluruh negara.</s>\n"
     ]
    }
   ],
   "source": [
    "generate_kwargs = dict(\n",
    "    inputs,\n",
    "    max_new_tokens=512,\n",
    "    top_p=0.95,\n",
    "    top_k=50,\n",
    "    temperature=0.9,\n",
    "    do_sample=True,\n",
    "    num_beams=1,\n",
    "    repetition_penalty=1.05,\n",
    ")\n",
    "r = model.generate(**generate_kwargs)\n",
    "print(tokenizer.decode(r[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6581db3c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': tensor([[    1,  1537,   282,   276, 28747, 22399, 12209,   276,  5311,  4499,\n",
       "          6635,   381,  1318, 28804,    13, 28798,  1067,  4209, 28747]],\n",
       "       device='cuda:0'), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]],\n",
       "       device='cuda:0')}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prompt = '<s>Soalan: kerajaan Madani bagus x?\\nJawapan:'\n",
    "inputs = tokenizer([prompt], return_tensors='pt', add_special_tokens=False).to('cuda')\n",
    "inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ddcf5217",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The history saving thread hit an unexpected error (OperationalError('database is locked')).History will not be written to the database.\n",
      "<s> Soalan: kerajaan Madani bagus x?\n",
      "Jawapan: soalan ni bagus. nak cakap bagus ke ape ke boleh cite sikit. Apa yang diaorang buat? Tapi dalam soalan ni la, ramai yang tak boleh jawab, lagi-lagi untuk soalan tu sahaja. Sebab soalan tu terlampau open ended dengan tiada langsung rujukan yang berkaitan. Hampir 99% gagal menjawab soalan ni. Dan saya memang respect kepada mereka yang bertungkus lumus untuk belajar. Tapi tahun ni sangat berbeza kerana mereka bertungkus-lumus untuk menghafal formula dan rumus. Kalau kita nak hafal semua, memang tidak mampu langsung untuk jawab soalan macam ni. Ramai yang patah balik apabila dapat soalan terbuka macam ni. Ada pula yang terbaca ayat dalam soalan, kemudian sebab dia terangkan terlalu detail tentang tu, dia terlepas pandang dan terabai soalan seterusnya. Huhu. Jangan main-main tau soalan ni, kalau jawab pun berdasarkan logik dan naluri, maka jawapannya adalah salah. So, pada mereka yang nak jawab soalan ni, saya cadangkan korang cuba baca soalan ni dulu, baru komen soalan ni mudah atau susah. Dan soalan ni, sebenarnya dah ada dalam banyak soalan percubaan sains sosial di luar sana. So, pada mereka yang masih tak tahu macam mana nak jawab, baik korang belajar dari situ. Ada pula yang suruh saya buatkan jawapan lengkap dengan bahan-bahan yang ada. Tetapi, saya cadangkan pada korang, jawapan ni bu\n"
     ]
    }
   ],
   "source": [
    "generate_kwargs = dict(\n",
    "    inputs,\n",
    "    max_new_tokens=512,\n",
    "    top_p=0.95,\n",
    "    top_k=50,\n",
    "    temperature=0.9,\n",
    "    do_sample=True,\n",
    "    num_beams=1,\n",
    "    repetition_penalty=1.05,\n",
    ")\n",
    "r = model.generate(**generate_kwargs)\n",
    "print(tokenizer.decode(r[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "79a538f9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': tensor([[    1,  1537,   282,   276, 28747,  8338,   314,   676, 28708,   307,\n",
       "           491,  4430, 20933, 18008,  7517,   314, 16179,  2794, 28718,    13,\n",
       "         28798,  1067,  4209, 28747]], device='cuda:0'), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]],\n",
       "       device='cuda:0')}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prompt = '<s>Soalan: macam mana nak install awscli dalam ubuntu\\nJawapan:'\n",
    "inputs = tokenizer([prompt], return_tensors='pt', add_special_tokens=False).to('cuda')\n",
    "inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ce51f3f4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<s> Soalan: macam mana nak install awscli dalam ubuntu\n",
      "Jawapan: Anda boleh memasang awscli dalam Ubuntu dengan menggunakan pengurus pakej apt-get. Caranya adalah seperti berikut: 1. Buka terminal dan kemas kini senarai pakej dengan menjalankan arahan berikut: ```sudo apt-get update``` 2. Pasang awscli dengan menjalankan arahan berikut: ```sudo apt-get install awscli``` Selepas menyelesaikan langkah-langkah di atas, anda akan mempunyai awscli dipasang dalam Ubuntu anda. Anda boleh menggunakannya untuk berinteraksi dengan AWS SDK dalam kod Python anda.</s>\n"
     ]
    }
   ],
   "source": [
    "generate_kwargs = dict(\n",
    "    inputs,\n",
    "    max_new_tokens=512,\n",
    "    top_p=0.95,\n",
    "    top_k=50,\n",
    "    temperature=0.9,\n",
    "    do_sample=True,\n",
    "    num_beams=1,\n",
    "    repetition_penalty=1.05,\n",
    ")\n",
    "r = model.generate(**generate_kwargs)\n",
    "print(tokenizer.decode(r[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "822f035b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
